#!/bin/bash
###
# auther:hongxing.fan
# description:put log file on hadoop
# date:2015-03-17 Tuesday
# file:Put_ssn.sh
###

stat(){
	end_time=`date '+%Y-%m-%d %H:%M:%S'`
	if [[ $1 -eq 0 ]];then
		echo "$2	$end_time	$3	$4	$5	$6	$7	$8	$9	success" >> $log
	else
		echo "$2	$end_time	$3	$4	$5	$6	$7	$8	$9	error" >> $log
		return 1
	fi
	return 0
}

operate_date=`date  -d "-1 day" +%Y%m%d`
if [[ $# -eq 1 ]]
then
	operate_date=$1
fi
#operate_date=$1
local_path="/home/work/workToHadoop/ssn_log"
hdfs_path="/user/hive/warehouse/pass_data.db/ssn_log"
put_ssn="/home/work/workToHadoop/log/put_ssn_${operate_date}.txt"
get_ssn="/home/work/workToHadoop/log/get_ssn_${operate_date}.txt"
rm $put_ssn
touch $put_ssn
log="/home/work/workToHadoop/log/${operate_date}.log"
# $1=$? $2=operate $3=source
source ~/.bashrc

tryTime=0
while true;do
	i=$(grep -c "ssn_log" $get_ssn)
	if [[ $i -eq 0 ]];then
		if [[ $tryTime -lt 120 ]];then
			tryTime=`expr $tryTime + 1`
			sleep 600 # 30minute
			continue
		else
			#stat 1 "put" "ssn_log" "dt=$operate_date" "tryTime=$tryTime"
			exit
		fi
	fi
	break
done

host=`hostname`
cd $local_path
## put
errors="/home/work/workToHadoop/log/put_errors.log"
echo "" > $errors
will_put=$(grep -vxFf $put_ssn $get_ssn)
len=${#will_put[0]}
#pids=$(ps axu | grep -E "Get_ssn.sh" | wc -l | awk '{print $1}')
pids=$(ps aux | grep "\-get" | grep "dt=$operate_date" | wc -l | awk '{print $1}')
while [[ $pids -gt 1 || $len -ne 0 ]]
do
	for f in ${will_put[0]}
	do
		#echo $f"=="
		#continue
		p_num=$(ps aux | grep "\-put" | grep "dt=$operate_date" | wc -l | awk '{print $1}')
		#p_num=$(ps axu | grep "Put_ssn.sh" | wc -l)
		while [[ $p_num -gt 11 ]];do
			sleep 5
			p_num=$(ps aux | grep "\-put" | grep "dt=$operate_date" | wc -l | awk '{print $1}')
			#p_num=$(ps axu | grep "Put_ssn.sh" | wc -l)
		done
		
		dir=$(echo $f | awk -F"ssn_log/|/atte" '{print $2}')
		tpl=$(echo $f | awk -F"tpl=|/dt=" '{print $2}')
		file=$(echo $f | awk -F"dt=$operate_date/" '{print $2}')
		`hdfs dfs -mkdir -p $hdfs_path/$dir`
		echo "$f" >> $put_ssn
		{
			begin_time=`date '+%Y-%m-%d %H:%M:%S'`
			try=0
			`hdfs dfs -put $f $hdfs_path/$dir 2>> $errors`
			s=$?
			flag=0
			while [[ $s -ne 0 && $try -lt 50 ]]
			do
				has=$(grep -Ec "$hdfs_path/$dir/$file.*File.*exists$" $errors)
				if [[ $has -eq 1 ]];then
					flag=1
					break
				fi
				sleep 5
				try=$(expr $try + 1)
				`hdfs dfs -put $f $hdfs_path/$dir`
				s=$?
			done
			if [[ $flag -eq 0 ]];then
				stat $s "$begin_time" "ssn_log" "$operate_date" "$host" "put" "" "$hdfs_path/$dir/$file" "$f"
			fi
			if [[ $flag -eq 0 && $s -eq 0 ]];then
				$(hive -e "use pass_data;alter table ssn_log add partition(tpl='$tpl',dt=$operate_date) location 'hdfs:///user/hive/warehouse/pass_data.db/ssn_log/tpl=$tpl/dt=$operate_date';")
			fi
			#stat $? "ssn_log"  "$operate_date" "$host" "partition" "" "$hdfs_path/$f" "$local_path/$f"
		} &
	done
	sleep 3
	pids=$(ps aux | grep "\-get" | grep "dt=$operate_date" | wc -l | awk '{print $1}')
	#pids=$(ps axu | grep -E "Get_ssn.sh" | wc -l | awk '{print $1}')
	will_put=$(grep -vxFf $put_ssn $get_ssn)
	len=${#will_put[0]}
done
wait

load="LOAD DATA LOCAL INFILE '/home/work/workToHadoop/log/${operate_date}.log' INTO TABLE pass_log FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'"
$(mysql -h10.195.82.35 -uroot -p"mysql" -D"pass_data" -e "$load")
